﻿import os
import sys
import Dumper
import urllib2
import xml.dom.minidom

URL_ROOT = 'http://www.taiwantestcentral.com/WordList/'

COL_WORD = u'\u5b57\u8a5e'
COL_DEF = u'\u4e2d\u6587\u91cb\u7fa9'
COL_LEVEL = u'\u7d1a\u5225'
COL_FREQ = u'\u8003\u984c'

PART_RANGE = [1,6,24]
for i in range(12,22): PART_RANGE.append(i)

GROUP_RANGE = [1,6,13,21]
SPLIT_FACTOR = range(50, 61)

SET_BASIC_SUBJECT = {}
SET_BASIC_SUBJECT['base'] = 'WordListBySubject.aspx?Section='
SET_BASIC_SUBJECT['range'] = range(24)
SET_BASIC_SUBJECT['title'] = Dumper.encodeStr('Basic 2000 主題')
SET_BASIC_SUBJECT['template'] = {'tags' : 'English, EFL, ESL, GSL, Vocabulary, 常用2000, 基考, 中考'}

SET_BASIC_PART = {}
SET_BASIC_PART['base'] = 'WordListByPart.aspx?MainCategoryID=2&Part='
SET_BASIC_PART['range'] = PART_RANGE
SET_BASIC_PART['title'] = Dumper.encodeStr('Basic 2000 詞性')
SET_BASIC_PART['group'] = {'key':COL_LEVEL,'range':GROUP_RANGE}
SET_BASIC_PART['template'] = {'tags' : 'English, EFL, ESL, GSL, Vocabulary, 常用2000, 基考, 中考'}

SET_BASIC_WORD = {}
SET_BASIC_WORD['base'] = 'BCTWordList.aspx?Sort=Frequency&CategoryID='
SET_BASIC_WORD['range'] = {
    '12':Dumper.encodeStr('基礎1200'),
    '11':Dumper.encodeStr('挑戰800')
}
SET_BASIC_WORD['title'] = Dumper.encodeStr('Basic 2000 詞表')
SET_BASIC_WORD['template'] = {'tags' : 'English, EFL, ESL, GSL, Vocabulary, 常用2000, 基考, 中考'}

SET_GEPT_PART = {}
SET_GEPT_PART['base'] = 'WordListByPart.aspx?MainCategoryID=4&Part='
SET_GEPT_PART['range'] = PART_RANGE
SET_GEPT_PART['title'] = Dumper.encodeStr('全民英檢')
SET_GEPT_PART['group'] = {'key':COL_LEVEL,'range':GROUP_RANGE}
SET_GEPT_PART['split'] = {'factor':SPLIT_FACTOR,'range':GROUP_RANGE}
SET_GEPT_PART['template'] = {'tags' : 'English, EFL, ESL, GSL, Vocabulary, GEPT, 社考'}

SET_ABILITY_PART = {}
SET_ABILITY_PART['base'] = 'WordListByPart.aspx?MainCategoryID=17&Part='
SET_ABILITY_PART['range'] = PART_RANGE
SET_ABILITY_PART['title'] = Dumper.encodeStr('學科能力測驗')
SET_ABILITY_PART['group'] = {'key':COL_LEVEL,'range':GROUP_RANGE}
SET_ABILITY_PART['split'] = {'factor':SPLIT_FACTOR,'range':GROUP_RANGE}
SET_ABILITY_PART['template'] = {'tags' : 'English, EFL, ESL, GSL, Vocabulary, 學測, 高考'}

SET_APPOINT_PART = {}
SET_APPOINT_PART['base'] = 'WordListByPart.aspx?MainCategoryID=14&Part='
SET_APPOINT_PART['range'] = PART_RANGE
SET_APPOINT_PART['split'] = {'factor':SPLIT_FACTOR,'range':GROUP_RANGE}
SET_APPOINT_PART['title'] = Dumper.encodeStr('指定科目考試')
SET_APPOINT_PART['group'] = {'key':COL_LEVEL,'range':GROUP_RANGE}
SET_APPOINT_PART['template'] = {'tags' : 'English, EFL, ESL, GSL, Vocabulary, 指考, 高考'}


def parseContent(content):
    doc = xml.dom.minidom.parseString(content.encode('utf-8'))
    trs = doc.getElementsByTagName('tr')
    thead = trs[0]
    ths = thead.getElementsByTagName('th')
    columns = list()
    for i in range(len(ths)):
        txt = Dumper.getText(ths[i], '').strip()
        if len(txt) != 0 :
            columns.append((txt, i))
    words = list()
    tbody = trs[1:]
    for tr in tbody:
        tds = tr.getElementsByTagName('td')
        word = dict()
        for (colname, colindex) in columns:
            word[colname] = Dumper.getText(tds[colindex], '').strip()
        words.append(word)
    return words
    
def getSet(setConfig):
    global URL_ROOT
    urlRange = setConfig['range'] if isinstance(setConfig['range'], list) else setConfig['range'].keys()
    dumper = Dumper.FileDumper(setConfig['title'])
    #dumper = Dumper.QuizletDumper(COL_WORD, COL_DEF, setConfig.get('template'))
    for i in urlRange:
        url = setConfig['base'] + str(i)
        #print url
        result = urllib2.urlopen(URL_ROOT + url)
        if result.getcode() == 200 :
            content = Dumper.encodeStr(result.read())
            if isinstance(setConfig['range'], dict):
                title = setConfig['range'][i]
            else:
                try:
                    title = content[content.index('<h2>')+len('<h2>'):content.index('</h2>')].strip()
                except ValueError:
                    title = ''
            #print title
            content = content[content.index('<table class="WordList'):content.index('</table>')+len('</table>')].replace('&', '&amp;')
            words = parseContent(content)
            Dumper.subConfigOutput(i, setConfig, words, title, COL_WORD, dumper)
                
if __name__ == '__main__':
    getSet(SET_BASIC_SUBJECT)
    getSet(SET_BASIC_PART)
    getSet(SET_BASIC_WORD)
    getSet(SET_GEPT_PART)
    #getSet(SET_ABILITY_PART)
    getSet(SET_APPOINT_PART)
